#!/local/usr/bin/python

import os, sys
import cnl2mc_console

class Validator:
    
    def __init__(self):
        self.console = cnl2mc_console.Console()
    def config(self, configFile):
        self.console.config(configFile)
    
    def getTestingSet(self, dataNodeDict, proportion = 0.1):
        total = len(dataNodeDict)        
        foldSize = int(total*proportion)
        testingNodeDict = {}
        dic = dataNodeDict.copy()
        ids = dataNodeDict.keys()
        import random
        random.seed(1)
        testingIdFold = random.sample(ids, foldSize)            
        for id in testingIdFold:
            testingNodeDict[id] = dic.pop(id)
            ids.remove(id)
        leftIds = ids
        return (testingNodeDict, leftIds)
    
    def getDevSet(self, dataNodeDict, leftIds, proportion = 0.0):
        total = len(dataNodeDict)
        foldSize = int(total*proportion)
        pass
        return
    
    def getTrainingSet(self, dataNodeDict, leftIds, proportion = 0.9):
        total = len(dataNodeDict)
        foldSize = int(total*proportion)
        trainingNodeDict = {}        
        dic = dataNodeDict.copy()
        import random
        random.seed(1)
        trainingIdFold = random.sample(leftIds, foldSize)            
        for id in trainingIdFold:
            trainingNodeDict[id] = dic.pop(id)
            leftIds.remove(id)
        return (trainingNodeDict, leftIds)
    
    
    def main(self):
        console = self.console
        sqlConfigDict = console.xmlConf.getSqlConfig()
        dataNodeDict = console.readTrainingDataNodeDict(sqlConfigDict)                                            
        console.preprocess(dataNodeDict)
        
        testProp = console.xmlConf.getTestProportion()
        (testingNodeDict, leftIds) = self.getTestingSet(dataNodeDict, testProp)
        trainProp = console.xmlConf.getTrainProportion()
        (trainingNodeDict, leftIds) = self.getTrainingSet(dataNodeDict, leftIds, trainProp)
        
        dirname = console.xmlConf.getGenerateFilePath()
        basename = os.path.basename(console.xmlConf.getFeatureMatrixFileName())
        outputPath = os.path.join(dirname,"training_%s_Instance"%str(len(trainingNodeDict)))   
        testingFeatureMatrixFileName = os.path.join(outputPath, "test_"+basename)             
        trainingFeatureMatrixFileName = os.path.join(outputPath, "train_"+basename)
        
        
        indexMethod = console.xmlConf.getIndexMethod()
        indexer = console.instFact.getNewIndexer(indexMethod)
        attachment = console.getIndexAttachment(indexer, trainingNodeDict)
        console.indexInFile(trainingFeatureMatrixFileName, trainingNodeDict, indexer, attachment)
        console.indexInFile(testingFeatureMatrixFileName, testingNodeDict, indexer, attachment)                                
        focusedClassesFilename = console.xmlConf.getFocusedClassesFilename()
        """
        # <exp>
        import histogram
        reportName = "../experiments/globalClassFreqDist.txt"
        histogram.reportFreqDistHistogram(console.globalClassFreqDist, reportName)
        print "wordAndClassHistogram...\n"
        focusedClassList = open(focusedClassesFilename, 'rU').read().split()
        trainingDataNodesIter = trainingNodeDict.itervalues()
        histogram.wordAndClassHistogram(console.globalWordFreqDist, focusedClassList, trainingDataNodesIter)
        # </exp>
        """      
        
        """ """
        if os.path.exists(focusedClassesFilename):
            focusedClassList = open(focusedClassesFilename, 'rU').read().split()
        else:
            focusedClassList = console.globalClassFreqDist.keys()        
        totalFileNumber = len(focusedClassList)
        printedFileNumber = 0
        print "totalFileNumber: %d"%totalFileNumber
        
        toFileType = console.xmlConf.getGenerateFileType()
                    
        reduceDimMethodType = console.xmlConf.getReduceDimMethodType()
        reduceDimMethodName = console.xmlConf.getReduceDimMethodName()
        freqThreshold = console.xmlConf.getFrequencyThreshold()
        IgThreshold = console.xmlConf.getInfoGainThreshold()
        
        for pickedClass in focusedClassList:
            binaryClassList = [pickedClass, '-1']
            reducedDimWordList = console.reduceDim(trainingNodeDict, binaryClassList, reduceDimMethodType, reduceDimMethodName, freqThreshold, IgThreshold)
            trainingFilename = "train_%s"%pickedClass
            # add IG_Reducing here. And a arff attributes parser
            testingFilename =  "test_%s"%pickedClass
            console.generateVectorFile(trainingNodeDict, trainingFeatureMatrixFileName, reducedDimWordList, toFileType, binaryClassList, outputPath, trainingFilename,False)
            console.generateVectorFile(testingNodeDict, testingFeatureMatrixFileName, reducedDimWordList, toFileType, binaryClassList, outputPath, testingFilename,False)
            printedFileNumber += 1
            print " %d file(s) in process: now is %d, -- %d file(s) left.\n" \
                    %(totalFileNumber, printedFileNumber, totalFileNumber - printedFileNumber)
                    
        os.remove(trainingFeatureMatrixFileName)
        os.remove(testingFeatureMatrixFileName)
        """ """
if __name__ == '__main__':    
    validator = Validator()
    configFile = sys.argv[1]
    validator.config(configFile)    
    validator.main()
    
    
    
    
    
    
    
    
    
    
    